#coding:utf-8
'''This module contains handler dealing with process in which we extract the content    of XML documents and reorganize it in a way the Stanford Segmenter and POStagger
   can further work with it so that we can reconstruct the XML structure easier.
'''

import xml.sax.handler
import re
from general.langconv import Converter
from general.stringoperations import Pattern

class DocContentHandler(xml.sax.handler.ContentHandler):
    '''This is the class dealing with doc in XML format.
       We try to extract the content of the doc and reorganize in the following
       way:
       Start
       Doc
       ID xxxxxxxxxx
       Type xxxx
       EndDoc
       Headline xxxxxx
       EndHEADLINE
       Dateline xxxxxx
       EndDateline
       Text
       P
       xxxxx
       xxxxx
       EndP
       EndText
       End
    '''
    def __init__(self, f, fileType="news", idft="ZhangSheng", split="::::", period_pattern=Pattern.end_of_sentence):
        #File object for recording parsed results of XML file.
        self.f = f
        #Two types of XML: news typed by 'traditon'; web typed by 'web'.
        self.type = fileType

        self.idft = idft    #Identifying tag
        self.split = split  #Splitting tag

        #Pattern used to match period.
        self.period_pattern = period_pattern
        #State used to record which tag we are in.
        #For web documents, when in POST substate is used to record which
        #subtag we are in to distinguish whether the characters belong to
        #subtag or POST.
        self.state = ""
        self.substate = ""
        #buffer used to store the content to be written to file.
        self.buffer = ""
        #Tradtion Chinese to Simplified Chinese converter
        self.converter = Converter('zh-hans')

    def startElement(self, name, attributes):
        if self.type == "news":
            self.traditionalStart(name, attributes)
        else:
            self.webStart(name, attributes)

    def characters(self, data):
        if self.type == "news":
            self.traditionalCharacters(data)
        else:
            self.webCharacters(data)

    def endElement(self, name):
        if self.type == "news":
            self.traditionalEnd(name)
        else:
            self.webEnd(name)

    def utf8write(self):
        '''Write the content in buffer to file and clear buffer.'''
        if self.buffer == "":
            return
        self.f.write(self.buffer + "\n")
        self.buffer = ""

    def isSen(self):
        '''Determine whether self.buffer contents a sentence.'''
        if re.search(self.period_pattern, self.buffer, flags=re.X):
            return True
        else:
            return False

    def processText(self, tag):
        '''Process the text stored in self.buffer.
           After processing, the text stored in buffer is realigned line by line.
        '''
        if self.buffer == "":
            return
        if self.isSen():
            lines = re.split(self.period_pattern, self.buffer, flags=re.X)
            self.buffer = ""
            zhs_line = ""
            for line in lines:
                if re.match(self.period_pattern, line, flags=re.X):
                    zhs_line += line
                    self.buffer += "%s-%s%s%s\n" \
                                    % (tag, self.idft, self.split, zhs_line)
                    zhs_line = ""
                elif line == "":
                    continue
                else:
                    zhs_line += line.strip()
            if zhs_line != "":
                self.buffer += "%s-%s%s%s\n" \
                        % (tag, self.idft, self.split, zhs_line)
        else:
            zhs_line = self.buffer.strip()
            self.buffer = "%s-%s%s%s\n" % (tag, self.idft, self.split, zhs_line)
        self.buffer = self.buffer[:-1]

    def processQuote(self, pre):
        '''Process the value of 'PREVIOUSPOST'.
        '''
        p = pre.split('\n')
        for line in p:
            self.buffer = line.strip()
            self.processText("QUOTELINE")
            self.utf8write()
        self.buffer = ""


    def traditionalStart(self, name, attributes):
        self.state = name
        if name == self.idft:
            self.buffer = self.idft
            self.utf8write()
        elif name == "DOC":
            self.buffer = "DOC-%s\n%s-%s%s%s\n%s-%s%s%s" \
                          % (self.idft, "id", self.idft, self.split,\
                 attributes['id'], 'type', self.idft,\
                 self.split, attributes['type'])
            self.utf8write()
        elif name == "HEADLINE" or name == "DATELINE":
            self.buffer = "%s-%s%s" % (name, self.idft, self.split)
        elif name == "TEXT" or name == "P":
            self.buffer = "%s-%s" % (name, self.idft)
            self.utf8write()

    def webStart(self, name, attributes):
        if self.state == "POST":
            self.substate = name
        else:
            self.state = name

        if name == self.idft:
            self.buffer = self.idft
            self.utf8write()
        elif name == "DOC" or name == "TEXT":
            self.buffer = "%s-%s" % (name, self.idft)
            self.utf8write()
        elif name == "DOCID":
            self.buffer = "id-%s%s" % (self.idft, self.split)
        elif name == "DOCTYPE":
            self.buffer = "SOURCE-%s%s%s\ntype-%s%s" \
              % (self.idft, self.split, attributes['SOURCE'],\
                 self.idft, self.split)
        elif name == "POST":
            self.buffer = "%s-%s" % (name, self.idft)
            self.utf8write()
        elif name == "DATETIME" or name == "HEADLINE" or name == "POSTER" \
             or name == "POSTDATE":
            self.buffer = "%s-%s%s" % (name, self.idft, self.split)
        elif name == "QUOTE":
            self.substate = "in"
            self.buffer = "%s-%s" %(name, self.idft)
            self.utf8write()
            self.processQuote(attributes['PREVIOUSPOST'])

    def traditionalCharacters(self, data):
        if self.state == "HEADLINE" or self.state == "DATELINE":
            self.buffer += data.strip()
        elif self.state == "P" or self.state == "TEXT":
            self.buffer += data.strip()

    def webCharacters(self, data):
        if self.state == "DOCID" or self.state == "DOCTYPE":
            self.buffer += data.strip()
        elif self.state == "DATETIME" or self.state == "HEADLINE" \
             or self.substate == "POSTER" or self.substate == "POSTDATE":
            self.buffer += data.strip()
        elif self.state == "POST" and self.substate == "in":
            if data == "\n":
                '''Because the text in web documents is relatively irregular.
                   We can only separate each paragraph by '\n' symbol.
                '''
                if self.buffer != "\n" and self.buffer != "":
                    self.processText("LINE")
                    self.utf8write()
                    self.buffer = ""
            else:
                self.buffer += data.strip()

    def traditionalEnd(self, name):
        if name == self.idft:
            self.buffer = "End%s" % (name)
            self.utf8write()
            self.buffer = ""
        elif name == "HEADLINE" or name == "DATELINE":
            self.buffer += "\nEnd%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
        elif name == "TEXT":
            self.processText("TLINE")
            self.utf8write()
            self.buffer = "End%s-%s" % (name, self.idft)
            self.utf8write()
        elif name == "P":
            self.processText("PLINE")
            self.utf8write()
            self.buffer = "End%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
        elif name == "DOC":
            self.buffer = "End%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
        else:
            print("End", name)

    def webEnd(self, name):
        if name == self.idft:
            self.buffer = "End%s" % (name)
            self.utf8write()
        elif name == "DOC":
            self.buffer = "End%s-%s" % (name, self.idft)
            self.utf8write()
        elif name == "DOCID" or name == "DOCTYPE":
            self.utf8write()
        elif name == "DATETIME" or name == "HEADLINE" or name == "POSTER":
            self.substate = ""
            self.buffer += "\nEnd%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
        elif name == "POSTDATE":
            self.substate = "in"
            self.buffer += "\nEnd%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
        elif name == "QUOTE":
            self.buffer = "End%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
        elif name == "POST":
            self.substate = ""
            self.state = ""
            self.processText("LINE")
            self.utf8write()
            self.buffer = "EndPOST-%s" % (self.idft)
            self.utf8write()
        elif name == "TEXT":
            self.buffer = "End%s-%s" % (name, self.idft)
            self.utf8write()
            self.buffer = ""
